# -*- coding: utf-8 -*-

import traceback, os, re
import workerpool
import commonlib, datetime, time
from MongoDbLog import MongoDbLog
from MongoModel import MongoModel


LOG = MongoDbLog('VnExpress_RV', 'debug')
MONGO_SERVER = '27.0.12.106'
MONGO_PORT = 27017
DBNAME = 'raovat'
COLLECTION = 'vnexpress'
MAX_PROCESS = 100
MAX_DUPLICATE = 10

def forceQuit():
    pid = os.getpid()
    os._exit(1)
    os.kill(pid, 9) # kill -9 pid tren linux

class VnExpressRVTBVP(MongoModel):
    
    url = 'http://vnexpress.net/rao-vat/2/dich-vu-thiet-bi-van-phong/{0}/'
    n_url = 'http://vnexpress.net/rao-vat/2/dich-vu-thiet-bi-van-phong/{0}/&d={1}'
    cities = {'Ha-noi': 'Hà Nội', 'HCM': 'TP Hồ Chí Minh', 'Cac-tinh-phia-bac': 'Các tỉnh phía Bắc', 'Cac-tinh-phia-nam': 'Các tỉnh phía Nam'}
    
    def __init__(self, host='localhost', port=27017):
        MongoModel.__init__(self, host, port)
    
    def standardizeTimeStr(self, timeStr):
        '''Thứ hai, 11/7/2011, 14:30 GMT+7'''
        try:
            pattern = r'.+,\s(\d+)/(\d+)/(\d+),\s(\d+):(\d+).+'
            preg = re.compile(pattern)
            m = preg.search(timeStr)
            if m:
                return datetime.datetime(day=int(m.group(1)), month=int(m.group(2)), 
                        year=int(m.group(3)), hour=int(m.group(4)), minute=int(m.group(5))) + datetime.timedelta(seconds=time.timezone)
        except:
            LOG.error(traceback.format_exc())
    
    def getDetail(self, url, title):
        LOG.debug('getDetail(url={0})'.format(url))
        data = []
        contact = []
        error = False
        try:
            db = self.connection[DBNAME]
            collection = db[COLLECTION]
            tree = commonlib.getXMLTree(url)
            if tree == None or tree == '': return
            contentNode = tree.xpath("//div[@class='des_ct']")
            postedDate = commonlib.getElementText(tree.xpath("//div[@class='detail_head_3']/span[contains(@style, 'float')]"))
            postedDate = self.standardizeTimeStr(postedDate)
            if len(contentNode) == 0: return
            content = contentNode[0]
            text = commonlib.getElementText(content, tail=0)
            if text != '': data.append({'type': 'text', 'data': text})
            for item in content.xpath("./*"):
                if item.text == None and item.tail == None:
                    text = commonlib.getElementText(item, descendant=1)
                    if text != '': data.append({'type': 'text', 'data': text})
                else:
                    if item.text != None:
                        text = commonlib.getElementText(item, tail=0, text=1)
                        if text != '': data.append({'type': 'text', 'data': text})
                    if item.tail != None:
                        text = commonlib.getElementText(item, tail=1, text=0)
                        if text != '': data.append({'type': 'text', 'data': text})
            # Phan lien he cua nguoi dang tin rao vat
            for item in tree.xpath("//div[@class='description_img']/div[@class='div_contact'][1]/*"):
                text = commonlib.getElementText(item, text=1, tail=1)
                if text != '': contact.append({'type': 'text', 'data': text})
            # print to test
            for line in data:
                print "{0}: {1}".format(line['type'], line['data'])
            if len(data) == 0: raise Exception('Không lấy được dữ liệu, url={0}'.format(url))
            collection.save({'hashUrl': commonlib.getMD5Hash(url),
                'title': title,
                'description': '',
                'content': data,
                'contact': contact,
                'newsLink': url,
                'update': postedDate,
                'source': 'vnexpress.net',
                'lastupdate': datetime.datetime.utcnow(),
                'timestamp': time.time(),
                'date': datetime.datetime.utcnow()
                })
        except:
            LOG.error(traceback.format_exc())
            error = True
        finally:
            return True if not error else False
    
    def getList(self, cityId):
        LOG.debug('getList(cityId={0})'.format(cityId))
        currentDuplicated = 0
        currentProcessed = 0
        try:
            url = self.url.format(cityId)
            db = self.connection[DBNAME]
            collection = db[COLLECTION]
            while currentProcessed < MAX_PROCESS and currentDuplicated < MAX_DUPLICATE:
                tree = commonlib.getXMLTree(url)
                if tree == '' or tree == None: return
                t_date = ''
                for item in tree.xpath("//ul[@class='ul_level2']/li[contains(@class, 'level2')]"):
                    if currentDuplicated > MAX_DUPLICATE or currentProcessed > MAX_PROCESS:
                        if currentProcessed > MAX_PROCESS:
                            LOG.info('Quá giới hạn {0} tin'.format(MAX_PROCESS))
                        if currentDuplicated > MAX_DUPLICATE:
                            LOG.info('Quá giới hạn trùng {0} tin'.format(MAX_DUPLICATE))
                        break
                    link = commonlib.getAttribText(item.xpath("./div[@class='titleRV']//a"), 'href')
                    if link == '': continue
                    link = commonlib.urlJoin(url, link)
                    isExist = collection.find_one({'hashUrl': commonlib.getMD5Hash(link)})
                    if isExist:
                        LOG.info('Already exist in database, _id: {0}'.format(isExist['_id']))
                        currentDuplicated += 1; continue
                    title = commonlib.getElementText(item.xpath("./div[@class='titleRV']//a"), descendant=1)
                    t_date = commonlib.getElementText(item.xpath("./div[@class='dateRV']"))
                    
                    print title
                    print link
                    if self.getDetail(link, title): currentProcessed += 1
                try:
                    month = t_date.split('/')[0]
                    day = t_date.split('/')[1]
                    year = datetime.datetime.now().year
                    t_date = "{0}/{1}/{2}".format(year, month, day)
                    url = self.n_url.format(cityId, t_date)
                except:
                    break
        except:
            LOG.error(traceback.format_exc())
        finally:
            LOG.info('currentProcessed={0}, currentDuplicated={1}'.format(currentProcessed, currentDuplicated))
            return
    
if __name__ == '__main__':
    
    LOG.info('start crawler raovat vnexpress')
    crawler = VnExpressRVTBVP(MONGO_SERVER, MONGO_PORT)
#    crawler.getList('Ha-Noi')
    try:
        pool = workerpool.WorkerPool(size=5)
        pool.map(crawler.getList, crawler.cities.keys())
        pool.shutdown()
        pool.wait()
    except:
        LOG.error(traceback.format_exc())
#    crawler.getDetail('http://vnexpress.net/rao-vat/2/dich-vu-thiet-bi-van-phong/502627215-do-muc-may-in-tai-nha.html')
    LOG.info('finished crawler raovat vnexpress')
    forceQuit()